package com.dcivision.dms.core;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.servlet.http.HttpServletRequest;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.impl.LogFactoryImpl;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.FSDirectory;

import com.dcivision.dms.bean.DmsDocument;
import com.dcivision.dms.bean.DmsDocumentDetail;
import com.dcivision.dms.bean.DmsLocMaster;
import com.dcivision.dms.bean.DmsRoot;
import com.dcivision.dms.dao.DmsDocumentDAObject;
import com.dcivision.dms.dao.DmsDocumentDetailDAObject;
import com.dcivision.dms.dao.DmsLocMasterDAObject;
import com.dcivision.dms.dao.DmsRootDAObject;
import com.dcivision.framework.SessionContainer;
import com.dcivision.framework.SystemParameterConstant;
import com.dcivision.framework.SystemParameterFactory;
import com.dcivision.framework.Utility;
import com.dcivision.lucene.FieldNames;
import com.dcivision.lucene.FieldTypes;
import com.dcivision.lucene.IndexConverter;
import com.dcivision.lucene.IndexManager;

/**
 * This class is to update or reindex lucene data.
 * @author          Lee Lv
 * @company         DCIVision Limited
 * @creation date   11/11/2006
 * @version         $Revision: 1.3 $
 */
public class LuceneUpdater {
  
  public static final String REVISION = "$Revision: 1.3 $";
  
  private Log log = new LogFactoryImpl().getInstance(this.getClass());
  
  private SessionContainer sessionContainer = null;
  
  private Connection conn = null;
  
  private String indexPath = null;
  
  private DmsDocumentDAObject dmsDocumentDAO = null;
  
  private DmsDocumentDetailDAObject dmsDocumentDetailDAO = null;
  
  private DmsLocMasterDAObject dmsLocMasterDAO = null;
  
  private DmsRootDAObject dmsRootDAO = null;
  
  private IndexSearcher indexSearcher = null;
  
  private int counter = 0;
  
  private int recordCount = 0;
  
  private boolean isLookForOldLucene = false;

  public LuceneUpdater(SessionContainer sessionContainer, Connection conn) throws Exception {
    this.sessionContainer = sessionContainer;
    this.conn = conn;
  }
  
  /**
   * Start update lucene.
   * @param request
   * @param backPath
   * @param isUpdateOld
   * @return
   * @throws Exception
   */
  public String startUpdate(HttpServletRequest request, String backPath, boolean isUpdateOld) throws Exception {
    // validate action
    File backupDir = new File(backPath);
    if (backupDir.exists() || backupDir.isFile()) {
      return "ERROR: backup folder had exist.";
    }
    String indexBasePath = SystemParameterFactory.getSystemParameter(SystemParameterConstant.INDEX_BASE_PATH);
    File indexDir = new File(indexBasePath);
    if (!indexDir.exists()) {
      return "ERROR: index base path is not exist.";
    }
    // backup index files.
    try {
      this.copyFolder(indexDir, backupDir);
      this.emptyFolder(indexDir);
    }catch (Exception ex) {
      return "ERROR: backup folder error"+ex;
    }
    
    try {
      this.isLookForOldLucene = isUpdateOld;
      this.indexPath = backPath;
      this.counter = 0;
      this.recordCount = this.getRecordCountFromDB();
      request.getSession().setAttribute(SystemUpdater.LUCENE_UPDATE_MESSAGE_UPDATE_COUNTER, "0");
      request.getSession().setAttribute(SystemUpdater.LUCENE_UPDATE_MESSAGE_UPDATE_RECORDCOUNT, String.valueOf(this.recordCount));
      this.reIndexLucene(request);
    }catch (Exception ex) {
      throw ex;
    }finally {
      if (this.indexSearcher!=null) indexSearcher.close();
    }
    return ""+this.recordCount;
  }
  
  /**
   * Backup old lucene file to backup path.
   * @param indexDir
   * @param backupDir
   * @throws Exception
   */
  private void copyFolder(File indexDir, File backupDir) throws Exception {
    backupDir.mkdirs();
    File[] files = indexDir.listFiles();
    for (int i=0; i<files.length; i++) {
      File file = files[i];
      if (file.isDirectory()) {
        File backupNewDir = new File(backupDir.getAbsolutePath()+"/"+file.getName()+"/");
        this.copyFolder(file, backupNewDir);
      }else {
        FileInputStream fis = null;
        FileOutputStream fos = null;
        try {
          fis = new FileInputStream(file);
          fos = new FileOutputStream(backupDir.getAbsolutePath()+"/"+file.getName());
          int bufferSize = 8192;
          byte[] buffer = new byte[bufferSize];
          int length = -1;
      
          while ( (length = fis.read(buffer, 0, bufferSize)) != -1) {
            fos.write(buffer, 0, length);
          }
          fos.flush();
        } catch (Exception ex) {
          throw ex;
        } finally {
          try {
            fos.close();
            fis.close();
          } catch (Exception ignore) {
          }
        }
      }
    }
  }
  
  /**
   * Clear lucene path before reindex.
   * @param indexDir
   */
  private void emptyFolder(File indexDir) {
    File[] files = indexDir.listFiles();
    for (int i=0; i<files.length; i++) {
      File file = files[i];
      file.delete();
    }
  }
  
  /**
   * Do index lucene
   * @param request
   * @throws Exception
   */
  private void reIndexLucene(HttpServletRequest request) throws Exception {
    String[] fieldNames = new String[] {
        "D.ID", "D.DOCUMENT_NAME", "D.DOCUMENT_TYPE", "D.ROOT_ID", "D.REFERENCE_NO",
        "D.DESCRIPTION", "D.RECORD_STATUS", "D.ITEM_STATUS", "D.UPDATER_ID", "D.UPDATE_DATE",
        "D.USER_DEF_1", "D.USER_DEF_2", "D.USER_DEF_3", "D.USER_DEF_4", "D.USER_DEF_5", 
        "D.USER_DEF_6", "D.USER_DEF_7", "D.USER_DEF_8", "D.USER_DEF_9", "D.USER_DEF_10", 
        "V.ID", "V.ITEM_SIZE",  
        "C.ID", "C.CONVERTED_NAME", "C.SEGMENT_NO", "C.EXT"
    };
    String[] fieldTypes = new String[] {
        "Integer", "String", "String", "Integer", "String",
        "String", "String", "String", "Integer", "Timestamp",
        "String", "String", "String", "String", "String", 
        "String", "String", "String", "String", "String", 
        "Integer", "Integer",
        "Integer", "String", "Integer", "String"
    };
    String orderBy = "D.ID ASC, V.ID DESC";
    
    //reindex document (lucene)
    Map basePathCache = new HashMap();
    IndexManager indexManager = null;
    try {
      indexManager = new IndexManager(this.sessionContainer, this.conn);
      
      int pageIndex = 0;
      int pageSize = 100;
      while (true) {
        if (pageIndex*pageSize > recordCount) break;
        
        List lstDocumentIds = new ArrayList();
        for (int i=0; i<pageSize; i++) {
          lstDocumentIds.add(new Integer((pageIndex*pageSize)+i));
        }
        
        Map mapDocuments = this.getDmsDocumentDAO().getMapIdToDVCFieldValueArrayList(lstDocumentIds, fieldNames, fieldTypes, orderBy);
        
        for (int i=0; i<lstDocumentIds.size(); i++) {
          Integer documentID = (Integer)lstDocumentIds.get(i);
          List lstDocuments = (List)mapDocuments.get(documentID);
          if (lstDocuments!=null && !lstDocuments.isEmpty()) {
            Object[] fieldValues = (Object[])lstDocuments.get(0);

            try {
              Integer rootID = (Integer)fieldValues[3];
              
              List documentDetailList = this.getDmsDocumentDetailDAO().getListByDocumentID(documentID);
              String content = this.getFullTextFromLucene(documentID);
              String basePath = (String)basePathCache.get(rootID);
              if (basePath==null) {
                basePath = this.getBasePath(rootID);
                basePathCache.put(rootID, basePath);
              }
              
              Document document = this.getDocumentByFullParam(fieldValues, content, documentDetailList, basePath);
              indexManager.addDocument(document);
              log.info("Success to add lucene : Document "+(String)fieldValues[1]+"("+fieldValues[0]+").");
            }catch (Exception ex) {
              log.warn("Document "+(String)fieldValues[1]+"("+fieldValues[0]+") fail to add lucene.", ex);
            }catch (OutOfMemoryError ex) {
              log.warn("Document "+(String)fieldValues[1]+"("+fieldValues[0]+") fail to add lucene by out of memory.", ex);
            }
            
          }
          //if ((++this.counter)%10 == 0) {
            this.counter++;
            request.getSession().setAttribute(SystemUpdater.LUCENE_UPDATE_MESSAGE_UPDATE_COUNTER, String.valueOf(this.counter));
          //}
        }
        log.info("INDEX :"+pageIndex*pageSize);
        pageIndex++;
      }
      
    }catch (Exception ex) {
      log.error(ex, ex);
      throw ex;
    }finally {
      indexManager.close(true);
    }
  }
  
  /**
   * Get max update records count.
   * @return
   * @throws Exception
   */
  private int getRecordCountFromDB() throws Exception {
    PreparedStatement psmt = null;
    ResultSet rs = null;
    try {
      int max = 0;
      String sql = "SELECT max(ID) FROM dms_document";
      psmt = conn.prepareStatement(sql, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
      rs = psmt.executeQuery();
      if (rs.next()) {
        max = rs.getInt(1);
      }
      return max;
    }catch (Exception ex) {
      throw ex;
    }finally {
      try {rs.close();}catch(Exception ignore){}finally {rs=null;}
      try {psmt.close();}catch(Exception ignore){}finally {psmt=null;}
    }
  }
  
  /**
   * Get lucene full text from old lucene index.
   * @param documentID
   * @return
   * @throws IOException
   */
  private String getFullTextFromLucene(Integer documentID) throws IOException {
    if (isLookForOldLucene) {
      Query termQuery = new TermQuery(new Term("DCI_UNIQUE_ID", "D_"+documentID.toString()));
      Hits hits = this.getIndexSearcher().search(termQuery);
      if (hits.length()>0) {
        return hits.doc(0).get("FULL_TEXT");
      }else {
        return null;
      }
    }else {
      Query termQuery = new TermQuery(new Term(FieldNames.UID, documentID.toString()));
      Hits hits = this.getIndexSearcher().search(termQuery);
      if (hits.length()>0) {
        return hits.doc(0).get(FieldNames.CONTENT);
      }else {
        return null;
      }
    }
  }
  
  /**
   * Get new document object by giving data.
   * @param fieldValues
   * @param content
   * @param dmsDocumentDetailList
   * @param basePath
   * @return
   * @throws Exception
   */
  private Document getDocumentByFullParam(Object[] fieldValues, String content, List dmsDocumentDetailList, String basePath) throws Exception {  
    Document document = new Document();
    // set index main information
    document.add(getField(FieldNames.UID, ((Integer)fieldValues[0]).toString()));
    document.add(getField(FieldNames.DOCUMENT_NAME, (String)fieldValues[1]));
    document.add(getField(FieldNames.DOCUMENT_NAME_SORT, (String)fieldValues[1]));
    document.add(getField(FieldNames.ROOT_ID, ((Integer)fieldValues[3]).toString()));
    document.add(getField(FieldNames.RECORD_STATUS, (String)fieldValues[6]));
    
    if (!Utility.isEmpty((String)fieldValues[4])) {
      document.add(getField(FieldNames.REFERENCE_NO, (String)fieldValues[4]));
    }
    if (!Utility.isEmpty((String)fieldValues[5])) {
      document.add(getField(FieldNames.DESCRIPTION, (String)fieldValues[5]));
    }
    if (!Utility.isEmpty((String)fieldValues[25])) {
      document.add(getField(FieldNames.EXTENSION, (String)fieldValues[25]));
    }
    if (!Utility.isEmpty(fieldValues[21])) {
      document.add(getField(FieldNames.ITEM_SIZE, ((Integer)fieldValues[21]).toString()));
    }
    if (!Utility.isEmpty((String)fieldValues[7])) {
      document.add(getField(FieldNames.ITEM_STATUS, (String)fieldValues[7]));
    }
    if (!Utility.isEmpty(fieldValues[8])) {
      document.add(getField(FieldNames.UPDATER_ID, ((Integer)fieldValues[8]).toString()));
    }
    if (!Utility.isEmpty(fieldValues[9])) {
      document.add(getField(FieldNames.UPDATE_DATE, ((Timestamp)fieldValues[9]).toString()));
    }
    
    // set default profile
    for (int i=0; i<10; i++) {
      if (!Utility.isEmpty((String)fieldValues[10+i])) {
        document.add(getField(FieldNames.DEFAULT_INDEXES+i, FieldNames.DEFAULT_INDEXES, (String)fieldValues[10+i]));
      }
    }
    
    //set physicalfile information
    String extension = (String)fieldValues[25];
    if (content!=null) {
      document.add(getField(FieldNames.CONTENT, content));
    }else if (!Utility.isEmpty(extension) && IndexConverter.isAllowConvert(extension) && this.isDocumentType((String)fieldValues[2])) {
      try {
        Integer segmentNo = (Integer)fieldValues[24];
        File file = new File(basePath+"/"+(segmentNo!=null ? "Document/segment"+segmentNo.toString()+"/" : "")+(String)fieldValues[23]);
        if (file.exists() && file.isFile()) {
          InputStream ins = null;
          try {
            ins = new FileInputStream(file);
            content = IndexConverter.getText(ins, extension);
            if (content!=null) document.add(getField(FieldNames.CONTENT, content));
          }catch (Exception ex) {
            log.warn("Document "+(String)fieldValues[1]+"("+fieldValues[0]+") convert to text failure.");
          }finally {
            try {ins.close();} catch(Exception ignore) {}finally {ins=null;}
          }
        }else {
          log.warn("Document "+(String)fieldValues[1]+"("+fieldValues[0]+") physical file is missing.");
        }
      }catch (Exception ex) {
        log.warn("Document "+(String)fieldValues[1]+"("+fieldValues[0]+") convet to text failure by unknown errror.");
        log.error(ex, ex);
      }
    }
      
    //set user defined profile
    if (dmsDocumentDetailList!=null && !dmsDocumentDetailList.isEmpty()) {
      String udfIndexDetailIds = "";
      for (int i=0; i<dmsDocumentDetailList.size(); i++) {
        DmsDocumentDetail detail = (DmsDocumentDetail)dmsDocumentDetailList.get(i);
        if (i==0) {
          document.add(getField(FieldNames.USER_DEFINED_INDEX_ID, detail.getUserDefinedFieldID().toString()));
        }
        udfIndexDetailIds += ((i==0?"" : " ")+detail.getUserDefinedFieldDetailID());
        String value = (detail.getFieldValue()!=null ? detail.getFieldValue() : (detail.getDateValue()!=null ? detail.getDateValue().toString() : (detail.getNumericValue()!=null ? detail.getNumericValue().toString():null)));
        if (!Utility.isEmpty(value)) {
          document.add(getField(FieldNames.USER_DEFINED_INDEXES+i, FieldNames.USER_DEFINED_INDEXES, value));
        }
      }
      document.add(getField(FieldNames.USER_DEFINED_INDEX_DETAIL_IDS, udfIndexDetailIds));
    }
    return document;
  }
  

  private String getBasePath(Integer rootID) throws Exception {
    return ((DmsLocMaster) this.getDmsLocMasterDAO().getObjectByID(((DmsRoot)getDmsRootDAO().getObjectByIDIgnoreRecordStatus(rootID)).getLocID())).getLocPath();
  }
  
  public Field getField(String fieldName, String fieldValue) throws Exception {
    return new Field(fieldName, fieldValue, FieldTypes.getStore(fieldName), FieldTypes.getIndex(fieldName), FieldTypes.getTermVector(fieldName));
  }
  
  public Field getField(String fieldName, String typeName, String fieldValue) throws Exception {
    return new Field(fieldName, fieldValue, FieldTypes.getStore(typeName), FieldTypes.getIndex(typeName), FieldTypes.getTermVector(typeName));
  }
  
  
  private boolean isDocumentType(String documentType) throws Exception {
    return (DmsDocument.DOCUMENT_TYPE.equals(documentType) || DmsDocument.COMPOUND_DOC_TYPE.equals(documentType) || DmsDocument.EMAIL_DOC_TYPE.equals(documentType) || DmsDocument.ORIGINSLITY_EMAIL_TYPE.equals(documentType));
  }
  
  private IndexSearcher getIndexSearcher() throws IOException {
    if (this.indexSearcher==null) {
      this.indexSearcher = new IndexSearcher(FSDirectory.getDirectory(indexPath, false));
    }
    return this.indexSearcher;
  }
  
  private DmsDocumentDAObject getDmsDocumentDAO() {
    if (this.dmsDocumentDAO==null) {
      this.dmsDocumentDAO = new DmsDocumentDAObject(this.sessionContainer, this.conn);
    }
    return this.dmsDocumentDAO;
  }

  private DmsDocumentDetailDAObject getDmsDocumentDetailDAO() {
    if (this.dmsDocumentDetailDAO==null) {
      this.dmsDocumentDetailDAO = new DmsDocumentDetailDAObject(this.sessionContainer, this.conn);
    }
    return this.dmsDocumentDetailDAO;
  }
  
  private DmsLocMasterDAObject getDmsLocMasterDAO() {
    if (this.dmsLocMasterDAO==null) {
      this.dmsLocMasterDAO = new DmsLocMasterDAObject(this.sessionContainer, this.conn);
    }
    return this.dmsLocMasterDAO;
  }
  
  private DmsRootDAObject getDmsRootDAO() {
    if (this.dmsRootDAO==null) {
      this.dmsRootDAO = new DmsRootDAObject(this.sessionContainer, this.conn);
    }
    return this.dmsRootDAO;
  }

  public int getCounter() {
    return counter;
  }

  public int getRecordCount() {
    return recordCount;
  }

}
